#define _CRT_SECURE_NO_DEPRECATE
#include <algorithm>
#include "rs_log.h"
#include "rs_grammar.h"
#include "rs_stream.h"

rs_grammar::rs_grammar (
        const rs_string& affix_cg_fname,
        const rs_string& proper_lc_fname,
        const rs_string& orthography)
{
    // parses few grammar files: en_affix_cg.rst, proper_lc.rst, to_rules.rst
    rsg_valid = true;
    rsg_dict = 0;
    rs_log_file (L"Initializing grammars");
    add_ography (affix_cg_fname, orthography);
    rs_string unchan_fname = proper_lc_fname;
    if (unchan_fname != L"") {
        rsg_unchan = new rs_table (unchan_fname);
        if (!rsg_unchan->is_valid()) {
            rsg_valid = false;
            return;
        }
    } else {
        rsg_unchan = 0;
    }
}


bool rs_grammar::init_og_pair (
    rs_pair& ogp,
    const rs_string& src_og_name,
    const rs_string& tgt_og_name,
    bool allow_same)
{ 
    rs_string rsp_src_name = upper_case(src_og_name);
    rs_string rsp_tgt_name = upper_case(tgt_og_name);
    ogp.rsp_src_name = rsp_src_name;
    ogp.rsp_tgt_name = rsp_tgt_name;
    rs_log_console (L"Preparing conversion from " + rsp_src_name
        + L" to " + rsp_tgt_name);

    // find the dictionary column indexes
    // all the source columns are added to the ogp.rsp_src_cix_v
    // the first target column is stored in ogp.rsp_tgt_cix
    ogp.rsp_tgt_cix = text_cix ();
    for (unsigned tix=0; tix<rsg_dict->get_col_count(); tix++) {
        rs_string col_name = upper_case(rsg_dict->get_col_name(any_cix(tix)));
        if (col_name.substr(0,3) != L"OR_")
            continue;
        col_name = col_name.substr(3);
        if (rsp_src_name.size() <= col_name.size()  &&
            rsp_src_name == col_name.substr(0, rsp_src_name.size()))
            ogp.rsp_src_cix_v.push_back (text_cix (tix));
        if (rsp_tgt_name.size() <= col_name.size()  &&
            rsp_tgt_name == col_name.substr(0, rsp_tgt_name.size())
            &&  !ogp.rsp_tgt_cix.is_valid())
            ogp.rsp_tgt_cix = text_cix (tix);
    }
    if (ogp.rsp_src_cix_v.size() == 0) {
        rs_report_error (L"Unknown source orthography: ", rsp_src_name);
        return false;
    }
    if (!ogp.rsp_tgt_cix.is_valid()) {
        rs_report_error (L"Unknown target orthography: ", rsp_tgt_name);
        return false;
    }

    // find the affix table indexes
    ogp.rsp_src_atix = rsp_src_name.substr(0,2) == L"TO" ? 0 : 1;
    ogp.rsp_tgt_atix = rsp_tgt_name.substr(0,2) == L"TO" ? 0 : 1;
    if (ogp.rsp_src_cix_v.size() == 1
        && !allow_same
        && ogp.rsp_src_cix_v[0] == ogp.rsp_tgt_cix) {
        rs_report_error (L"Source and target orthography can not be equal: ",
            ogp.rsp_src_name);
        return false;
    }
    ogp.freql_cix = rsg_dict->get_numb_cix (L"FREQL");
    if (!ogp.freql_cix.is_valid()) {
        rs_report_error (L"Missinig column FREQL in dictionary");
        return false;
    }
    return true;
}



void rs_grammar::add_ography (
        const rs_string& affix_cg_fname,
        rs_string abbreviation)
{
    // adds specified orthography into list of known grammars if not yet listed
    // loads rules and dictionary if not yet loaded
    // returns true if orthography is available (either loaded now or already in the past)
    to_upper (abbreviation);
    unsigned tix;
    for (tix=0;  tix<rsg_affix.size();  tix++) {
        if (rsg_affix[tix].get_og_name() == abbreviation)
            // already loaded
            return;
    }
    rs_string fname1 = rs_affix::get_rules_fname (L"TO");
    rs_string fname2 = rs_affix::get_rules_fname (abbreviation);
    rs_string dict_fname, dict_msg;
    dict_fname = abbreviation + L"_dict.rst";
    dict_msg = L", " + dict_fname;
    if (file_exists (fname1)) {
        rs_log_console (L"Loading grammar rules: grammar="+abbreviation + L", file="+fname1);
        rsg_affix.push_back (rs_affix (affix_cg_fname, abbreviation));
    } else {
        rs_report_error (L"No valid rules table, " + fname1 + L" doesn't exist");
        rsg_valid = false;
        return;
    }
    if (file_exists (fname2)) {
        rs_log_console (L"Loading grammar rules: grammar="+abbreviation + L", file="+fname2);
        rsg_affix.push_back (rs_affix (affix_cg_fname, L"TO"));
    } else {
        rs_report_error (L"No valid rules table, " + fname2 + L" doesn't exist");
        rsg_valid = false;
        return;
    }
    rsg_og_count = rsg_affix.size();
    tix = rsg_og_count - 1;
    if (!rsg_affix[tix].is_valid()) {
        rsg_valid = false;
        return;
    }
    rsg_affix[tix].set_og_name (abbreviation);
    rsg_dict = new rs_table (dict_fname);
    rs_log_console (L"Loaded dictionary "+ dict_fname);
    rs_table& this_rst= *rsg_dict;
    if (!this_rst.is_valid()) {
        rsg_valid = false;
        return;
    }
    check_dictionary(tix);
}


rs_grammar::~rs_grammar ()
{
    if (rsg_unchan) {
        delete rsg_unchan;
    }
    rsg_unchan = 0;
}

bool rs_grammar::translate_any (
    const rs_string &src_word,
    text_cix src_cix,
    text_cix tgt_cix,
    numb_cix freql_cix,
    rs_string &tgt_word,
    bool& several) 
    const
{
    if (rsg_unchan) {
        rs_rix rix = rsg_unchan->get_text_row_any (src_word, text_cix (0), several);
        if (rix.is_valid()) {
            tgt_word = src_word;
            return true;
        }
    }
    bool result(true);
    several = false;
    rs_string cur_word (src_word); // currently converted word
    result = rsg_dict->translate_one (
        src_word, src_cix, freql_cix, tgt_cix, tgt_word, several);
    if (result)
        cur_word = tgt_word;
    return result;
}


void rs_grammar::check_dictionary (unsigned tix)
{
    // list words that have characters out of specified character set
    rs_table& dict_rst= *rsg_dict;
    rs_string abbr = dict_rst.get_abbr();
    rs_string valid_str = rsg_affix[0].get_alphabet();
    rs_string columns = L"OR_TO_US,OR_TO_UK,OR_SP"; // these text columns are checked
    rs_string_v col_v;
    string_to_vec (columns, col_v, L",");

    bool is_valid;
    rs_log_file (L"Checking words with unspecified characters in dictionary: ",
        abbr);
    for (rs_rix rix=rs_rix::begin(); rix<dict_rst.get_row_count(); rix++) {
        for (unsigned i=0; i<col_v.size(); i++)
        {
            text_cix tcix = dict_rst.get_text_cix(col_v[i]);
            const rs_string& word = dict_rst.get_text (tcix, rix);
            is_valid = has_valid_chars (word, valid_str);
            if (!is_valid) {
                rs_log_file (L"unspecified characters: ", word);
            }
        }
    }
    rs_log_file (L"Finished checking words with unspecified characters in dictionary: ",
        abbr);
}


bool rs_grammar::has_valid_chars (const rs_string& word, const rs_string& valid) const
{
    // if 'word' has characters not listed in 'valid', returns false
    // otherwise it returns false
    unsigned len=unsigned(word.length());
    for (unsigned i=0; i<len; i++) {
        if (valid.find (word[i]) == string::npos)
            return false;
    }
    return true;
}


bool rs_grammar::convert_normalized_word (
    const rs_string& src_word,
    rs_string& tgt_word,
    bool check_composition,
    bool logging,
    rs_statistics& stat,
    const rs_pair& ogp)
{
    // converts any lower-case word
    // returns true/false depending on success of conversion
    // todo improve handling of unknown words
    size_t src_len = src_word.length();
    if (src_len==0) {
        tgt_word = src_word;
        return false;
    }
    if (is_unchanged (src_word)) {
        // todo: cutspell has many proper names in dictionary, supress usage of them
        tgt_word = src_word;
        if (logging) {
            rs_log_file (L"unchanged:  " + src_word);
            stat.increase (rse_dict);
        }
        return true;
    }
    bool several, result;

    // first try to convert complete word
    int src_count = ogp.rsp_src_cix_v.size();
    for (int i=0; i<src_count; i++)
    {
        result = translate_any(src_word,
            ogp.rsp_src_cix_v[i], 
            ogp.rsp_tgt_cix,
            ogp.freql_cix,
            tgt_word, several);
        if (result)
        {
            return true;
        }
    }

    vector<size_t> af_vec;
    wstring stem_src, stem_tgt;
    const rs_affix& src_affix = rsg_affix[ogp.rsp_src_atix];
    text_cix split_found_cix; // column index where split to affixes was found
    for (int i=0; i<src_count; i++)
    {
        result = src_affix.split_affixes (src_word, stem_src, af_vec,
            rsg_dict->get_column_ref(ogp.rsp_src_cix_v[i]));
        if (result) {
            split_found_cix = text_cix (i);
            break;
        }
    }
    if (result && af_vec.size() > 0 && logging) {
        rs_log_file (L"split_affixes: " + src_word + L"->" + src_affix.show_affixes (
            src_word, stem_src, af_vec));
    }
    if (!result && check_composition) {
        result = convert_composed_word (src_word, tgt_word, logging, stat, ogp);
        if (result)
            return true;
    }
    if (!result) {
        if (logging) {
            rs_log_file (L"unknown:    " + src_word);
            bool several;
            if (!stat.cnv_missing.get_text_row_any (src_word, text_cix(0), several).is_valid())
                stat.cnv_missing.append_text (src_word, text_cix(0));
            stat.increase (rse_unknown);
        }
        tgt_word = src_word;
        if (stat.cnv_show_unknown)
            tgt_word += L"*";
        return false;
    }

    translate_any(stem_src,split_found_cix, ogp.rsp_tgt_cix,ogp.freql_cix,
        stem_tgt, several);
    // todo: avoid double look-up when calling 'translate_any', remember data in split_affixes
    const rs_affix& tgt_affix = rsg_affix[ogp.rsp_tgt_atix];
    result = tgt_affix.apply_affixes (tgt_word, stem_tgt, af_vec);
    if (!result) {
        tgt_word = src_word;
        if (stat.cnv_show_unknown)
            tgt_word += L"#";
        rs_log_file (L"unkn.n affix: " + src_word);
        return false;
    }
    if (several && stat.cnv_show_unknown)
        tgt_word+= L'+';
    if (af_vec.size() == 0) {
        if (logging) {
            rs_log_file (L"dictionary: " + src_word + L" to " + tgt_word);
            stat.increase(rse_dict);
        }
    } else {
        if (logging) {
            rs_log_file (L"stemmed:    " + src_word + L" to " + tgt_word);
            stat.increase(rse_stemmed);
        }
    }
    return true;
}


bool rs_grammar::convert_composed_word (
    const rs_string& src,
    rs_string& cnv,
    bool logging,
    rs_statistics& stat,
    const rs_pair& ogp)
{
    size_t src_len = src.length();
    if (src_len < 4) {
        cnv = src;
        return false;
    }
    for (size_t ix=2; ix <= src_len-2; ix++) {
        // trying to split 'src' at position 'ix'
        rs_string src1 (src.substr (0, ix));
        rs_string cnv1;
        if (!convert_normalized_word (src1, cnv1, false, false, stat, ogp))
            continue;
        rs_string src2 (src.substr (ix));
        rs_string cnv2;
        if (!convert_normalized_word (src2, cnv2, false, false, stat, ogp))
            continue;
        // found composition
        cnv = cnv1 + cnv2;
        if (logging) {
            rs_log_file (L"split:      " + src + L" to "
                + src1 + L"/" + cnv1
                + L", " + src2 + L"/" + cnv2
                );
            stat.increase (rse_dict);
        }
        return true;
    }
    cnv = src;
    return false;
}
